#encoding=utf8

'''
批量清洗文本数据
'''
import ClearStopWord,EachFile,os


if __name__ == '__main__':
	path = os.path.abspath(r'../CSCMNews')
	for i,obj in enumerate(EachFile.loadFiles(path)):
		if i%10000 == 0:
			file_path = os.path.join(os.path.join(path,obj[0]),obj[1])
			file_txt = ClearStopWord.read_file(file_path)
			file_txt = ClearStopWord.clear_for_cut(file_txt)
			print(file_txt)